# -*- coding: utf-8 -*-
import datetime
import os
import logging

BOT_NAME = 'jiage315'
ENV_NAME = 'dev'

SPIDER_MODULES = ['jiage315.spiders']
NEWSPIDER_MODULE = 'jiage315.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'jiage315 (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
COOKIES_ENABLED = True
today = datetime.datetime.now()
log_dir = 'logs/{}'.format(str(today.year) + str(today.month))
if not os.path.exists(log_dir):
    os.makedirs(log_dir)

def switchTime(sec, what):
    _time = datetime.datetime.now() - datetime.timedelta(hours=8)
    return _time.timetuple()
logging.Formatter.converter = switchTime

LOG_FILE = None
# LOG_FILE = '{}/log-{}.log'.format(log_dir, ENV_NAME)
LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: ['+ENV_NAME+',hosp-notice] %(message)s'
LOG_LEVEL = 'INFO'

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
SPIDER_MIDDLEWARES = {
    'jiage315.middlewares.HospNewsSpiderMiddleware': 543,
}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
# pipelines数值越小,越先执行
ITEM_PIPELINES = {
    'jiage315.pipelines.HospNewsPipeline': 300,
}

# 数据库url
DB_URL = 'mysql+pymysql://root:12345678@localhost:3306/oil?charset=utf8mb4'

DOWNLOADER_MIDDLEWARES_BASE = {
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
}

# 打开重试开关
RETRY_ENABLED = True
# 重试次数
RETRY_TIMES = 10
# 超时
DOWNLOAD_TIMEOUT = 1000
# 延时
DOWNLOAD_DELAY = 3
HTTPERROR_ALLOWED_CODES = [500, 502, 503, 504, 508, 400, 403, 404, 408, 302]
RETRY_HTTP_CODES = [500, 502, 503, 504, 508, 400, 403, 404, 408, 302]


# 最小发布日期，小于此日期的数据不取
MIN_PUB_DATE = '2022-02-01'
